# %%
import urllib
import requests
import time
from lxml import etree

ap_top_news_url = 'https://apnews.com/hub/ap-top-news'
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67'
request_headers = {'User-Agent': ua}


def get_top_news_url(index=0):
    wb_data = requests.get(
        ap_top_news_url, headers=request_headers, verify=False)
    root = etree.HTML(wb_data.text)
    a_xpath = f'//main/div[1]/div[2]/div[{index+1}]//a[1]'
    img_path = f'//main/div[1]/div[2]/div[{index+1}]//img'

    a = root.xpath(a_xpath)[0]
    img = root.xpath(img_path)[0]
    img_url = img.get('src')
    href = a.get('href')
    top_1_news_url = urllib.parse.urljoin('https://apnews.com/', href)
    return top_1_news_url, img_url


def get_top_news_article(top_news_url, html_file_output_file, img_url=None):
    wb_data = requests.get(top_news_url, headers=request_headers)
    root = etree.HTML(wb_data.text)
    title_xpath = "//h1"
    img_xpath = '//main/div[1]/bsp-carousel//img'
    para_xpath = "//main/div[2]/div[2]/p"

    title = root.xpath(title_xpath)[0].text
    img_url = root.xpath(img_xpath)[0].get('src') if root.xpath(
        img_xpath) and not root.xpath(img_xpath)[0].get('src').startswith('data:') else img_url
    paragraphs_el = root.xpath(para_xpath)

    doc_template = open('article_template.html', 'r', encoding='utf-8').read()
    doc_template = doc_template.replace('%HeadTitle%', title)
    today = time.strftime('%Y-%m-%d', time.localtime())
    doc_template = doc_template.replace('%Date%', today)
    try:
        if img_url is not None:
            doc_template = doc_template.replace('%ImgSrc%', img_url)
    except:
        pass
    p_template = '''<p class="Component-root-0-2-194 p Component-p-0-2-176">{}</p>\n'''

    paragraphs = ''
    for p in paragraphs_el:
        _els =[tx for tx in p.itertext()]
        paragraphs += p_template.format(''.join(_els))

    doc_template = doc_template.replace('%paragraphs%', paragraphs)
    with open(html_file_output_file, 'w', encoding='utf-8') as f:
        f.write(doc_template)
    return title
# %%


if __name__ == '__main__':
    import sys
    index = 0
    if len(sys.argv) > 1:
        index = int(sys.argv[1])
    top_news_url, img_url = get_top_news_url(index)
    print(top_news_url, img_url)
    get_top_news_article(top_news_url, 'test.html', img_url)
